#
# Organizes estimates from determinants of tracking
#

#==============================================
# Preamble
#==============================================
START_TIME = Sys.time()

scripts.dir <- dirname(sys.frame(1)$ofile)
print(scripts.dir)
setwd(scripts.dir)

setwd('..')
main.dir = getwd()

data.dir = paste(scripts.dir, 'working', sep='/')
out.dir = paste(scripts.dir, 'output', sep='/')

library('reshape')
library('dplyr')
library('latex2exp')
library('stringr')

options(stringsAsFactors=FALSE)

#==============================================
# read in estimates
#==============================================

print(data.dir)
setwd(data.dir)

files.0 = data.frame(f.n=list.files())
files.0$one = 1

for(ab in c('A', 'B')){
  reg.0 = paste('^txtr_4-h.', ab, sep='')
  files.1 = files.0[grepl(reg.0, files.0$f.n), ]
  
  data.10 = NULL
  for(i in 1:nrow(files.1)){
    fn = files.1$f.n[i]
    data.0 = as.matrix(read.csv(fn))
    data.0[,] = gsub('^=', '', data.0[,])
    data.1 = data.0[2:(nrow(data.0)-5), ]
    colnames(data.1) = c('x', data.0[1, 2:ncol(data.0)])
    
    obs.0 = as.data.frame(t(data.0[nrow(data.0)-(5:2), 2:ncol(data.0)]))
    colnames(obs.0) = c('nobs', 'r2', 'ar2', 'nclust')
    obs.0$one = 1
    obs.0$spec = data.0[1, 2:ncol(data.0)]
    
    a = str_match(obs.0$spec,
                  '([a-z0-9_-]{1,10})\\s+([a-z0-9_-]{1,10})\\s+([a-z0-9_-]{1,10})\\s+([a-z0-9_-]{1,10})')
    obs.0$hyp = a[, 2]
    obs.0$target = a[, 3]
    obs.0$sample = a[, 4]
    obs.0$fe = a[, 5]
    
    data.1 = as.data.frame(data.1)
    
    for(j in 1:nrow(data.1)){
      if(trimws(data.1[j, 'x']) != ''){
        data.1 = data.1[j:nrow(data.1), ]
        break
      }
    }
    
    for(j in 1:nrow(data.1)){
      if(trimws(data.1[j, 'x']) == ''){
        data.1[j, 'x'] = data.1[j-1, 'x']
        data.1[j, 'var'] = 'se'
      }else{
        data.1[j, 'var'] = 'b'
      }
    }
    data.2 = as.data.frame(melt(data.1, id.vars=c('x', 'var')))
    names(data.2) = c('x', 'stat', 'spec', 'val')
    
    data.3 = as.data.frame(merge(data.2, obs.0, by='spec', all.x=T))
    
    data.3$f.n = fn
    
    data.10 = rbind(data.10, data.3)
  }
  
  data.10$x = trimws(data.10$x)
  
  h.0 = c('det_7 t_unadj all fe-none', 'det_8 t_unadj all fe-none',
          'det_9 t_unadj all fe-none', 'det_10 t_unadj all fe-none',
          'det_10 t_unadj all fe-dg_id',
          'det_10 t_unadj all fe-cg_id', 'det_6 t_unadj all fe-cg_id')
  x.0 = c('cgmnzlm', 'cgsdzlm', 'cflmagnet', 'cflchart', 'cntydempa0', 'dedge_priv0', 'cghisp',
          'cgblack', 'cgasian', 'cgother', 'cgdisadv', 'cglep')
  tab4.10 = data.frame(one=rep(1, (length(x.0) * 3)+3))
  for(target in c('t_unadj', 't_rel')){
    for(elem.mid in c('all', 'elem', 'middle')){
      h.3 = gsub('all', elem.mid, gsub('t_unadj', target, h.0))
      a = with(data.10, (spec %in% h.3) & (x %in% x.0))
      tab5.0 = data.10[a, ]
      tab5.0 = tab5.0[!duplicated(with(tab5.0, paste(spec, x, stat, sep='.'))), ]
      tab5.0$val = paste('="', tab5.0$val, '"', sep='')
      tab5.1 = as.data.frame(cast(tab5.0, x + spec ~ stat, value='val'))
      tab5.1$z = ''
      tab5.2 = as.data.frame(melt(tab5.1, id.vars=c('x', 'spec'), value='val'))
      
      a = 1:length(x.0)
      names(a) = x.0
      tab5.2$x.id = a[tab5.2$x]
      tab5.3 = as.data.frame(cast(tab5.2, x.id + x + variable ~ spec, value='val'))
      tab5.3$variable = as.character(tab5.3$variable)
      tab5.3 = tab5.3[, c('x.id', 'x', 'variable', h.3)]
      
      obs.20 = tab5.0[!duplicated(tab5.0$spec), ]
      rownames(obs.20) = obs.20$spec
      
      tab5.4 = obs.20[c(NA, NA, NA, h.3), c('r2', 'nobs', 'nclust')]
      rownames(tab5.4) = c('x.id', 'x', 'variable', h.3)
      
      tab5.5 = rbind(tab5.3, t(tab5.4))
      a = names(tab5.5)
      tab5.5 = cbind(c(paste('measure type:', target),
                       paste('elem/mid/all:', elem.mid),
                       rep('', nrow(tab5.5)-2)), tab5.5)
      names(tab5.5) = c('specification', a)
      tab5.5[, 'blank'] = ''
      
      tab4.10 = cbind(tab4.10, tab5.5)
    }
  }
  
  x.1 = c('cgmnzlm', 'cgsdzlm',
          'cglep', 'lep_con',
          'cgdis_phy', 'cgdis_oth', 'cgspedr',
          'cggifted', 'curric_diff', 'cgclsszavg', 'ctexpa')
  x.2 = c(x.1)
  h.2 = c('pol_1 t_unadj all fe-none', 'pol_2 t_unadj all fe-none',
          'pol_3 t_unadj all fe-none', 'pol_3 t_unadj all fe-dg_id',
          'pol_3 t_unadj all fe-cg_id', 'pol_4 t_unadj all fe-cg_id')
  tab5.10 = data.frame(one=rep(1, (length(x.2) * 3)+3))
  for(target in c('t_unadj', 't_rel')){
    for(elem.mid in c('all', 'elem', 'middle')){
      h.3 = gsub('all', elem.mid, gsub('t_unadj', target, h.2))
      a = with(data.10, (spec %in% h.3) & (x %in% x.2))
      tab5.0 = data.10[a, ]
      tab5.0 = tab5.0[!duplicated(with(tab5.0, paste(spec, x, stat, sep='.'))), ]
      tab5.0$val = paste('="', tab5.0$val, '"', sep='')
      tab5.1 = as.data.frame(cast(tab5.0, x + spec ~ stat, value='val'))
      tab5.1$z = ''
      tab5.2 = as.data.frame(melt(tab5.1, id.vars=c('x', 'spec'), value='val'))
      
      a = 1:length(x.2)
      names(a) = x.2
      tab5.2$x.id = a[tab5.2$x]
      tab5.3 = as.data.frame(cast(tab5.2, x.id + x + variable ~ spec, value='val'))
      tab5.3$variable = as.character(tab5.3$variable)
      tab5.3 = tab5.3[, c('x.id', 'x', 'variable', h.3)]
      
      obs.20 = tab5.0[!duplicated(tab5.0$spec), ]
      rownames(obs.20) = obs.20$spec
      
      tab5.4 = obs.20[c(NA, NA, NA, h.3), c('r2', 'nobs', 'nclust')]
      rownames(tab5.4) = c('x.id', 'x', 'variable', h.3)
      
      tab5.5 = rbind(tab5.3, t(tab5.4))
      a = names(tab5.5)
      tab5.5 = cbind(c(paste('measure type:', target),
                       paste('elem/mid/all:', elem.mid),
                       rep('', nrow(tab5.5)-2)), tab5.5)
      names(tab5.5) = c('specification', a)
      
      tab5.10 = cbind(tab5.10, tab5.5)
    }
  }
  
  print(out.dir)
  setwd(out.dir)
  
  if(ab == 'A'){
    write.csv(tab4.10, 'txtr_5-t4-det.csv', na='', row.names=F)
    write.csv(tab5.10, 'txtr_5-t5-pol.csv', na='', row.names=F)
  }else{
    write.csv(tab4.10, 'txtr_5-t-det-drop.csv', na='', row.names=F)
    write.csv(tab5.10, 'txtr_5-t-pol-drop.csv', na='', row.names=F)
  }
}


#==============================================
# Postamble
#==============================================

END_TIME = Sys.time()
print("Time elapsed in txtr_5.r:")
print(END_TIME - START_TIME)


